# -*- encoding: utf-8 -*-
import csv
import os
import re
import difflib
import ssl
import time
from urllib.request import urlopen
from urllib.error import URLError
from jinja2 import Template
from lxml import etree

from .manual_data import manual_merges, manual_splits

# 行政区变更情况: http://xzqh.mca.gov.cn/description?dcpid=2021
DISTRICT_URL = [
    ('1980', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708040959.html'),
    ('1981', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708041004.html'),
    ('1982', 'http://www.mca.gov.cn/article/sj/tjbz/a/1980-2000/201707141125.html'),
    ('1983', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708160821.html'),
    ('1984', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220856.html'),
    ('1985', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220858.html'),
    ('1986', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220859.html'),
    ('1987', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220902.html'),
    ('1988', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220903.html'),
    ('1989', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708041017.html'),
    ('1990', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708041018.html'),
    ('1991', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708041020.html'),
    ('1992', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220910.html'),
    ('1993', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708041023.html'),
    ('1994', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220911.html'),
    ('1995', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220913.html'),
    ('1996', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220914.html'),
    ('1997', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220916.html'),
    ('1998', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220918.html'),
    ('1999', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220921.html'),
    ('2000', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220923.html'),
    ('2001', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220925.html'),
    ('2002', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220927.html'),
    ('2003', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220928.html'),
    ('2004', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220930.html'),
    ('2005', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220935.html'),
    ('2006', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220936.html'),
    ('2007', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220939.html'),
    ('2008', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220941.html'),
    ('2009', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220943.html'),
    ('2010', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220946.html'),
    ('2011', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201707271552.html'),
    ('2012', 'http://www.mca.gov.cn/article/sj/tjbz/a/201713/201707271556.html'),
    ('2013', 'https://files2.mca.gov.cn/cws/201404/20140404125552372.htm'),
    ('2014', 'https://files2.mca.gov.cn/cws/201502/20150225163817214.html'),
    ('2015', 'http://www.mca.gov.cn/article/sj/tjbz/a/2015/201706011127.html'),
    ('2016', 'http://www.mca.gov.cn/article/sj/tjbz/a/2016/201612/201705311652.html'),
    ('2017', 'http://www.mca.gov.cn/article/sj/tjbz/a/2018/201803131439.html'),
    ('2018', 'http://www.mca.gov.cn/article/sj/xzqh/2018/201804-12/20181011221630.html'),
    ('2019', 'http://www.mca.gov.cn/article/sj/xzqh/1980/2019/202002281436.html'),
    ('2020', 'http://www.mca.gov.cn/article/sj/xzqh/2020/20201201.html'),
]


class CodeSpider:
    def __init__(self):
        self.district_code = dict()
        self.code_change = dict()
        self.name_change = dict()
        self.unaccounted = dict()
        self.latest_code = dict()

    @staticmethod
    def __parse_html(response) -> list:
        """
        解析网页内容
        :param response:
        :return:
        """

        # resp_text = etree.tostring(etree.parse('id_tools/test.html', etree.HTMLParser())) # 用于测试解析
        def check_td(el):
            # text = el.text
            text = td.xpath('string(.)').strip()  # 提取被标签包裹的文本内容
            if el is not None and text:
                return text.strip()
            return False

        # [('110000', '北京市')]
        code_list = list()
        try:
            html = etree.HTML(response)
            all_tr = html.xpath('//tr')

            for tr in all_tr[2:]:
                if len(tr) < 2:
                    continue
                td_list = list()
                for td in tr.xpath('td'):
                    td_text = check_td(td)
                    if not td_text or td_text == '行政区划代码' or td_text == '行政区划名称':
                        continue
                    td_list.append(td_text)

                if len(td_list) == 2:
                    code_list.append(tuple(td_list))

            return code_list
        except IndexError:
            print('获取页面元素失败')

    def __district_code_spider(self, timeline=2):
        """
        抓取中华人民共和国民政部的区域代码
        :return:
        """
        context = ssl._create_unverified_context()
        # context = ssl.create_default_context(cafile=certifi.where())
        for url in DISTRICT_URL:
            try:
                resp = urlopen(url[1], context=context)
                resp_text = resp.read().decode('utf-8')
                year = url[0]
                parse_data = self.__parse_html(resp_text)
                self.__write_csv(year, parse_data)
                print(f'正在抓取{year}年的区域代码，地址：{url}')
            except URLError:
                print('URL Error!!')
            time.sleep(timeline)

    @staticmethod
    def __extract_name(fullname):
        name = fullname
        # name = re.sub(r'(市|县|区|地区)$', '', name)
        # name = re.sub(
        #     r'((汉|阿昌|白|保安|布朗|布依|朝鲜|达斡尔|傣|德昂|侗|东乡|独龙|鄂伦春|俄罗斯|鄂温克|高山|仡佬|哈尼|哈萨克|赫哲|回|基诺|京|景颇|柯尔克孜|拉祜|黎|傈僳|珞巴|满|毛南|门巴|蒙古|苗|仫佬|纳西|怒|普米|羌|撒拉|畲|水|塔吉克|塔塔尔|土|土家|佤|维吾尔|乌兹别克|锡伯|瑶|彝|裕固|藏|壮)族)+自治',
        #     '', name)
        return name

    def __change_dict(self, year: int, deleted_list: list, new_add_list: list):
        """
        将对比出变化的区域代码转换成字典
        :param year:
        :param deleted_list:
        :param new_add_list:
        :return:
        """
        additions = dict()
        for add in new_add_list:
            try:
                code, full_name = add.split(',')
                name = self.__extract_name(full_name)
                if name not in additions:
                    additions[name] = [code, name]
                    additions[code] = [code, name]
            except ValueError:
                pass
        # code_change: {'1981': [{'old_code': 12345, 'new_code':'123', 'old_name':'456','new_name': '456'}]}
        # name_change:{'1981': [{'code': 12345, 'old_name':'123', 'new_name':'456'}]}
        for dl in deleted_list:
            try:
                deleted_code, full_name = dl.split(',')
            except ValueError:
                continue

            if year in manual_merges() and deleted_code in manual_merges()[year]:
                continue
            if year in manual_splits() and deleted_code in manual_splits()[year]:
                continue

            deleted_name = self.__extract_name(full_name)
            if deleted_name in additions:
                added_code, added_name = additions[deleted_name]
                # endswith("00") ??
                attr = {
                    'old_code': deleted_code,
                    'old_name': deleted_name,
                    'new_code': added_code,
                    'new_name': added_name
                }
                if year not in self.code_change:
                    self.code_change[year] = []
                self.code_change[year].append(attr)
                # 更新区域代码中的开始和结束时间
                for dc in self.district_code.get(added_code, []):
                    if dc.get('address') == added_name and year > 1980:
                        dc['start_year'] = year
                for dc in self.district_code.get(deleted_code, []):
                    if dc.get('address') == deleted_name:
                        dc['end_year'] = year - 1

            elif deleted_code in additions:
                # {'1981': [{'code': 12345, 'old_name':'123', 'new_name':'456'}]}
                attr = {
                    'code': deleted_code,
                    'old_name': full_name,
                    'new_name': additions[deleted_code][1]
                }
                if year not in self.name_change:
                    self.name_change[year] = []
                self.name_change[year].append(attr)

            else:
                if year not in self.unaccounted:
                    self.unaccounted[year] = []
                self.unaccounted[year].append({
                    'code': deleted_code,
                    'name': deleted_name,
                })

    @staticmethod
    def __write_csv(filename, parse_data):
        with open(f'id_tools/csv_data/{filename}.csv', 'w+', encoding='utf8', newline='') as f:
            writer = csv.writer(f)
            writer.writerow(['code', 'name'])
            writer.writerows(parse_data)

    @staticmethod
    def __read_csv(filename: str) -> list:
        reader_list = list()
        with open(f'id_tools/csv_data/{filename}.csv', 'r', encoding='utf8') as f:
            reader = csv.reader(f)
            for r in reader:
                reader_list.append(','.join(r))
        return reader_list

    def __render(self):
        attr = {
            'latest_code': self.latest_code,
            'all_district_code': self.district_code,
            'code_change': self.code_change,
            'name_change': self.name_change,
            'unaccounted': self.unaccounted
        }
        with open('id_tools/data_py.template', 'r', encoding='utf8') as f:
            template = Template(f.read()).render(attr)

        with open('id_tools/generate_data.py', 'w', encoding='utf8') as f:
            f.write(template)

    def __parse_to_dict(self, parse_data):
        # district_code格式：{'110000': [{'address': '北京市', 'start_year': '', 'end_year': ''}],}
        for code, name in parse_data:
            attr = {
                'address': name,
                'start_year': '',
                'end_year': '',
            }
            if code not in self.district_code:
                self.district_code[code] = [attr]
            else:
                dc_all_name = [dc.get('address', '') for dc in self.district_code[code]]
                if name not in dc_all_name:
                    self.district_code[code].append(attr)

    def __diff_file(self, filename_list):
        """
        比较每年的区域代码变化
        :return:
        """
        enum_filename = list(enumerate(filename_list))
        for index, year in enum_filename:
            if year == filename_list[-1]:
                continue
            before_data = self.__read_csv(str(year))
            after_year = enum_filename[index + 1][1]
            after_data = self.__read_csv(str(after_year))

            diff_data = difflib.unified_diff(before_data, after_data)
            deleted_list = list()
            new_add_list = list()
            print('-----------分割线-------------')
            for dd in list(diff_data)[4:]:
                if dd.startswith('-'):
                    deleted_list.append(dd.split('-')[1].strip())
                elif dd.startswith('+'):
                    new_add_list.append(dd.split('+')[1].strip())
            print(deleted_list)
            print(new_add_list)
            self.__change_dict(int(year) + 1, deleted_list, new_add_list)

    def convert_to_dict(self):
        """
        将本地的CSV转换成字典并写入文件
        :return:
        """
        filename_list = sorted([f.split('.')[0] for f in os.listdir('id_tools/csv_data')])
        enum_filename = list(enumerate(filename_list))
        for index, year in enum_filename:
            address_list = list()
            csv_data = self.__read_csv(str(year))
            for cd in csv_data[1:]:
                cd_split = cd.split(',')
                address_list.append(cd_split)

            self.__parse_to_dict(address_list)
            if year == filename_list[-1]:
                # {'110000': '北京市', '110101': '东城区',}
                for code in address_list:
                    self.latest_code[code[0]] = code[1]

        self.__diff_file(filename_list)
        self.__render()

    def spider(self, timeline=2, local=False):
        if not local:
            self.__district_code_spider(timeline=timeline)
        self.convert_to_dict()


code_spider = CodeSpider()
